{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "search for wiki online"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from gensim.summarization.summarizer import summarize\n",
    "gen_summary=summarize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tkinter as tk\n",
    "\n",
    "# define the function that will be used to process the input\n",
    "def process_input(input_str):\n",
    "    # this is just a simple example function that reverses the input string\n",
    "    return input_str[::-1]\n",
    "\n",
    "# define the function that will be called when the submit button is pressed\n",
    "def submit_input():\n",
    "    # get the text that was entered into the input box\n",
    "    input_text = input_box.get()\n",
    "\n",
    "    # process the input using the predefined function\n",
    "    output_text = process_input(input_text)\n",
    "\n",
    "    # display the output in the output box\n",
    "    output_box.configure(state='normal')\n",
    "    output_box.delete(1.0, tk.END)\n",
    "    output_box.insert(tk.END, output_text)\n",
    "    output_box.configure(state='disabled')\n",
    "\n",
    "# create the main tkinter window\n",
    "root = tk.Tk()\n",
    "root.title(\"Input Processing\")\n",
    "\n",
    "# create the input box\n",
    "input_label = tk.Label(root, text=\"Enter some text:\")\n",
    "input_label.pack()\n",
    "input_box = tk.Entry(root)\n",
    "input_box.pack()\n",
    "\n",
    "# create the submit button\n",
    "submit_button = tk.Button(root, text=\"Submit\", command=submit_input)\n",
    "submit_button.pack()\n",
    "\n",
    "# create the output box with a vertical scroll bar\n",
    "output_label = tk.Label(root, text=\"Output:\")\n",
    "output_label.pack()\n",
    "output_frame = tk.Frame(root)\n",
    "output_frame.pack()\n",
    "output_scrollbar = tk.Scrollbar(output_frame)\n",
    "output_scrollbar.pack(side='right', fill='y')\n",
    "output_box = tk.Text(output_frame, height=10, width=30, yscrollcommand=output_scrollbar.set, state='disabled')\n",
    "output_box.pack(side='left', fill='both', expand=True)\n",
    "output_scrollbar.config(command=output_box.yview)\n",
    "\n",
    "# start the main loop\n",
    "root.mainloop()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_wordcloud(input_text):\n",
    "    # tokenize the input text into words\n",
    "    words = nltk.word_tokenize(input_text)\n",
    "\n",
    "    # generate the word cloud\n",
    "    wordcloud = WordCloud(width=800, height=400, background_color=\"white\").generate(' '.join(words))\n",
    "\n",
    "    # create a matplotlib figure to display the word cloud\n",
    "    plt.figure(figsize=(8, 4))\n",
    "    plt.imshow(wordcloud, interpolation='bilinear')\n",
    "    plt.axis(\"off\")\n",
    "\n",
    "    # display the word cloud in a tkinter window\n",
    "    wordcloud_window = tk.Toplevel(root)\n",
    "    wordcloud_window.title(\"Word Cloud\")\n",
    "    wordcloud_canvas = tk.Canvas(wordcloud_window, width=800, height=400)\n",
    "    wordcloud_canvas.pack()\n",
    "    wordcloud_canvas.create_image(0, 0, anchor='nw', image=wordcloud_image)\n",
    "\n",
    "    # save the word cloud image as a PNG file\n",
    "    wordcloud_image = plt.gcf().canvas.get_tk_widget().tk.photoimage(format='png')\n",
    "    wordcloud_image.write(\"wordcloud.png\", format=\"png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "from io import BytesIO\n",
    "\n",
    "\n",
    "def generate_wordcloud(text):\n",
    "    # Generate WordCloud image\n",
    "    wordcloud = WordCloud(width=800, height=400, background_color=\"white\").generate(text)\n",
    "\n",
    "    # Create a buffer to hold the image data\n",
    "    buffer = BytesIO()\n",
    "\n",
    "    # Save the image to the buffer in JPEG format\n",
    "    plt.imshow(wordcloud, interpolation='bilinear')\n",
    "    plt.axis(\"off\")\n",
    "    plt.savefig(buffer, format='jpeg')\n",
    "\n",
    "    # Reset the buffer position to the start so it can be read from\n",
    "    buffer.seek(0)\n",
    "\n",
    "    # Return the JPEG image data\n",
    "    return buffer.read()\n",
    "\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# wordcloud code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tkinter as tk\n",
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "\n",
    "import tkinter as tk\n",
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "\n",
    "def generate_wordcloud(text):\n",
    "    # Generate WordCloud image\n",
    "    wordcloud = WordCloud(width=800, height=400, background_color=\"white\").generate(text)\n",
    "    \n",
    "    # Save the image to a permanent file location as JPEG\n",
    "    file_path = os.path.join(os.getcwd(), \"wordcloud.jpeg\")\n",
    "    wordcloud.to_file(file_path)\n",
    "    \n",
    "    # Return the file path\n",
    "    return file_path\n",
    "\n",
    "from PIL import Image, ImageTk\n",
    "\n",
    "def generate_and_show_wordcloud():\n",
    "    # Get the text from the Text widget\n",
    "    text = text_widget.get(\"1.0\", tk.END)\n",
    "    \n",
    "    # Generate WordCloud image\n",
    "    image_path = generate_wordcloud(text)\n",
    "    \n",
    "    # Convert the image to a format that can be shown by Tkinter\n",
    "    image = Image.open(image_path)\n",
    "    image = image.convert(\"RGB\")\n",
    "    photo = ImageTk.PhotoImage(image)\n",
    "    \n",
    "    # Load the image and display it on a Canvas widget\n",
    "    canvas.create_image(0, 0, anchor=tk.NW, image=photo)\n",
    "    canvas.image = photo\n",
    "\n",
    "\n",
    "# Create the main Tkinter window\n",
    "root = tk.Tk()\n",
    "root.title(\"WordCloud Generator\")\n",
    "\n",
    "# Create a Text widget for inputting text\n",
    "text_widget = tk.Text(root)\n",
    "text_widget.pack()\n",
    "\n",
    "# Create a Button widget to generate the WordCloud image\n",
    "button = tk.Button(root, text=\"Generate WordCloud\", command=generate_and_show_wordcloud)\n",
    "button.pack()\n",
    "\n",
    "# Create a Canvas widget for displaying the WordCloud image\n",
    "canvas = tk.Canvas(root, width=800, height=400)\n",
    "canvas.pack()\n",
    "\n",
    "# Start the main event loop\n",
    "root.mainloop()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tkinter as tk\n",
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from PIL import Image, ImageTk\n",
    "\n",
    "\n",
    "def generate_wordcloud_gui():\n",
    "    # Create the main Tkinter window\n",
    "    root = tk.Tk()\n",
    "    root.title(\"WordCloud Generator\")\n",
    "\n",
    "    # Create a Text widget for inputting text\n",
    "    text_widget = tk.Text(root)\n",
    "    text_widget.pack()\n",
    "\n",
    "    # Create a Button widget to generate the WordCloud image\n",
    "    button = tk.Button(root, text=\"Generate WordCloud\",\n",
    "                       command=lambda: generate_and_show_wordcloud(text_widget, canvas))\n",
    "    button.pack()\n",
    "\n",
    "    # Create a Canvas widget for displaying the WordCloud image\n",
    "    canvas = tk.Canvas(root, width=800, height=400)\n",
    "    canvas.pack()\n",
    "\n",
    "    # Start the main event loop\n",
    "    root.mainloop()\n",
    "\n",
    "\n",
    "def generate_and_show_wordcloud(text_widget, canvas):\n",
    "    # Get the text from the Text widget\n",
    "    text = text_widget.get(\"1.0\", tk.END)\n",
    "\n",
    "    # Generate WordCloud image\n",
    "    wordcloud = WordCloud(width=800, height=400, background_color=\"white\").generate(text)\n",
    "\n",
    "    # Save the image to a permanent file location as JPEG\n",
    "    file_path = os.path.join(os.getcwd(), \"wordcloud.jpeg\")\n",
    "    wordcloud.to_file(file_path)\n",
    "\n",
    "    # Convert the image to a format that can be shown by Tkinter\n",
    "    image = Image.open(file_path)\n",
    "    image = image.convert(\"RGB\")\n",
    "    photo = ImageTk.PhotoImage(image)\n",
    "\n",
    "    # Load the image and display it on a Canvas widget\n",
    "    canvas.create_image(0, 0, anchor=tk.NW, image=photo)\n",
    "    canvas.image = photo\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_wordcloud_gui()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tkinter as tk\n",
    "from PIL import Image, ImageTk\n",
    "\n",
    "# Create the main Tkinter window\n",
    "root = tk.Tk()\n",
    "\n",
    "# Open the JPEG image file\n",
    "image_path = \"wordcloud.jpg\"\n",
    "image = Image.open(image_path)\n",
    "\n",
    "# Convert the image to a Tkinter-compatible format\n",
    "tk_image = ImageTk.PhotoImage(image)\n",
    "\n",
    "# Create a Label widget to display the image\n",
    "label = tk.Label(root, image=tk_image)\n",
    "label.pack()\n",
    "\n",
    "# Start the main event loop\n",
    "root.mainloop()\n",
    "# Example usage\n",
    "text = \"Lorem ipsum dolor sit amet, consectetur adipiscing elit.\"\n",
    "image_data = generate_wordcloud(text)\n",
    "# Do something with the image data, such as save it to a file or display it in a GUI"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# google search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from googlesearch import search\n",
    "import requests\n",
    "import time\n",
    "\n",
    "def search_query(query):\n",
    "    # define headers to use in the request\n",
    "    headers = {\n",
    "        \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3\"\n",
    "    }\n",
    "    \n",
    "    for j in search(query, tld=\"co.in\", num=1, stop=2, pause=5):\n",
    "        url = j\n",
    "        time.sleep(5)\n",
    "    \n",
    "    return url\n",
    "\n",
    "\n",
    "# ========== 2. Using urllib & BeatifulSoup ==========\n",
    "# Import packages\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "\n",
    "# Specify url of the web page\n",
    "source = urlopen(url).read()\n",
    "\n",
    "# Make a soup \n",
    "soup = BeautifulSoup(source,'lxml')\n",
    "soup\n",
    "\n",
    "# Extract the plain text content from paragraphs\n",
    "paras = []\n",
    "for paragraph in soup.find_all('p'):\n",
    "    paras.append(str(paragraph.text))\n",
    "\n",
    "# Extract text from paragraph headers\n",
    "heads = []\n",
    "for head in soup.find_all('span', attrs={'mw-headline'}):\n",
    "    heads.append(str(head.text))\n",
    "\n",
    "# Interleave paragraphs & headers\n",
    "text = [val for pair in zip(paras, heads) for val in pair]\n",
    "text = ' '.join(text)\n",
    "\n",
    "# Drop footnote superscripts in brackets\n",
    "text = re.sub(r\"\\[.*?\\]+\", '', text)\n",
    "\n",
    "# Replace '\\n' (a new line) with '' and end the string at $1000.\n",
    "text = text.replace('\\n', '')[:-11]\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Create accountLog in\n",
      " Create account Log in\n",
      "ContributionsTalk\n",
      "Main pageContentsCurrent eventsRandom articleAbout WikipediaContact usDonate\n",
      "HelpLearn to editCommunity portalRecent changesUpload file\n",
      "What links hereRelated changesUpload fileSpecial pagesPermanent linkPage informationCite this pageWikidata item\n",
      "Download as PDFPrintable version\n",
      "Wikimedia CommonsWikiversity\n",
      "Contents\n",
      "\n",
      "\n",
      "\n",
      "(Top)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "1History\n",
      "\n",
      "\n",
      "\t\t\t\tToggle History subsection\n",
      "\t\t\t\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "1.1Symbolic NLP (1950s – early 1990s)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "1.2Statistical NLP (1990s–2010s)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "1.3Neural NLP (present)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "2Methods: Rules, statistics, neural networks\n",
      "\n",
      "\n",
      "\t\t\t\tToggle Methods: Rules, statistics, neural networks subsection\n",
      "\t\t\t\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "2.1Statistical methods\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "2.2Neural networks\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3Common NLP tasks\n",
      "\n",
      "\n",
      "\t\t\t\tToggle Common NLP tasks subsection\n",
      "\t\t\t\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3.1Text and speech processing\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3.2Morphological analysis\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3.3Syntactic analysis\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3.4Lexical semantics (of individual words in context)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3.5Relational semantics (semantics of individual sentences)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3.6Discourse (semantics beyond individual sentences)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3.7Higher-level NLP applications\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "4General tendencies and (possible) future directions\n",
      "\n",
      "\n",
      "\t\t\t\tToggle General tendencies and (possible) future directions subsection\n",
      "\t\t\t\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "4.1Cognition and NLP\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "5See also\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "6References\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "7Further reading\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "8External links\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "1.1Symbolic NLP (1950s – early 1990s)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "1.2Statistical NLP (1990s–2010s)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "1.3Neural NLP (present)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "2.1Statistical methods\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "2.2Neural networks\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3.1Text and speech processing\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3.2Morphological analysis\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3.3Syntactic analysis\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3.4Lexical semantics (of individual words in context)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3.5Relational semantics (semantics of individual sentences)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3.6Discourse (semantics beyond individual sentences)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3.7Higher-level NLP applications\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "4.1Cognition and NLP\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Natural language processing\n",
      "AfrikaansالعربيةAzərbaycancaবাংলাBân-lâm-gúБеларускаяБеларуская (тарашкевіца)БългарскиBosanskiCatalàČeštinaDanskDeutschEestiΕλληνικάEspañolEsperantoEuskaraفارسیFrançaisGalego한국어Հայերենहिन्दीHrvatskiBahasa IndonesiaIsiZuluÍslenskaItalianoעבריתಕನ್ನಡქართულიLietuviųМакедонскиमराठीمصرىМонголမြန်မာဘာသာ日本語ଓଡ଼ିଆPicardPiemontèisPolskiPortuguêsRomânăRuna SimiРусскийSimple EnglishکوردیСрпски / srpskiSrpskohrvatski / српскохрватскиSuomiதமிழ்ไทยTürkçeУкраїнськаTiếng Việt粵語中文\n",
      "ArticleTalk\n",
      "\n",
      "ReadEditView history\n",
      "ReadEditView history\n",
      "Natural language processing (NLP) is an interdisciplinary subfield of linguistics, computer science, and artificial intelligence concerned with the interactions between computers and human language, in particular how to program computers to process and analyze large amounts of natural language data.  The goal is a computer capable of \"understanding\" the contents of documents, including the contextual nuances of the language within them. The technology can then accurately extract information and insights contained in the documents as well as categorize and organize the documents themselves.\n",
      "\n",
      "Challenges in natural language processing frequently involve speech recognition, natural-language understanding, and natural-language generation.\n",
      "\n",
      "History[edit]\n",
      "Natural language processing has its roots in the 1950s. Already in 1950, Alan Turing published an article titled \"Computing Machinery and Intelligence\" which proposed what is now called the Turing test as a criterion of intelligence, though at the time that was not articulated as a problem separate from artificial intelligence. The proposed test includes a task that involves the automated interpretation and generation of natural language.\n",
      "\n",
      "Symbolic NLP (1950s – early 1990s)[edit]\n",
      "The premise of symbolic NLP is well-summarized by John Searle's Chinese room experiment: Given a collection of rules (e.g., a Chinese phrasebook, with questions and matching answers), the computer emulates natural language understanding (or other NLP tasks) by applying those rules to the data it confronts.\n",
      "\n",
      "1950s: The Georgetown experiment in 1954 involved fully automatic translation of more than sixty Russian sentences into English. The authors claimed that within three or five years, machine translation would be a solved problem.[2]  However, real progress was much slower, and after the ALPAC report in 1966, which found that ten-year-long research had failed to fulfill the expectations, funding for machine translation was dramatically reduced.  Little further research in machine translation was conducted in America (though some research continued elsewhere, such as Japan and Europe[3]) until the late 1980s when the first statistical machine translation systems were developed.\n",
      "1960s: Some notably successful natural language processing systems developed in the 1960s were SHRDLU, a natural language system working in restricted \"blocks worlds\" with restricted vocabularies, and ELIZA, a simulation of a Rogerian psychotherapist, written by Joseph Weizenbaum between 1964 and 1966.  Using almost no information about human thought or emotion, ELIZA sometimes provided a startlingly human-like interaction. When the \"patient\" exceeded the very small knowledge base, ELIZA might provide a generic response, for example, responding to \"My head hurts\" with \"Why do you say your head hurts?\".\n",
      "1970s: During the 1970s, many programmers began to write \"conceptual ontologies\", which structured real-world information into computer-understandable data.  Examples are MARGIE (Schank, 1975), SAM (Cullingford, 1978), PAM (Wilensky, 1978), TaleSpin (Meehan, 1976), QUALM (Lehnert, 1977), Politics (Carbonell, 1979), and Plot Units (Lehnert 1981).  During this time, the first chatterbots were written (e.g., PARRY).\n",
      "1980s: The 1980s and early 1990s mark the heyday of symbolic methods in NLP. Focus areas of the time included research on rule-based parsing (e.g., the development of HPSG as a computational operationalization of generative grammar), morphology (e.g., two-level morphology[4]), semantics (e.g., Lesk algorithm), reference (e.g., within Centering Theory[5]) and other areas of natural language understanding (e.g., in the Rhetorical Structure Theory). Other lines of research were continued, e.g., the development of chatterbots with Racter and Jabberwacky. An important development (that eventually led to the statistical turn in the 1990s) was the rising importance of quantitative evaluation in this period.[6]\n",
      "Statistical NLP (1990s–2010s)[edit]\n",
      "Up to the 1980s, most natural language processing systems were based on complex sets of hand-written rules.  Starting in the late 1980s, however, there was a revolution in natural language processing with the introduction of machine learning algorithms for language processing.  This was due to both the steady increase in computational power (see Moore's law) and the gradual lessening of the dominance of Chomskyan theories of linguistics (e.g. transformational grammar), whose theoretical underpinnings discouraged the sort of corpus linguistics that underlies the machine-learning approach to language processing.[7]\n",
      "\n",
      "1990s: Many of the notable early successes on statistical methods in NLP occurred in the field of machine translation, due especially to work at IBM Research, such as IBM alignment models.  These systems were able to take advantage of existing multilingual textual corpora that had been produced by the Parliament of Canada and the European Union as a result of laws calling for the translation of all governmental proceedings into all official languages of the corresponding systems of government.  However, most other systems depended on corpora specifically developed for the tasks implemented by these systems, which was (and often continues to be) a major limitation in the success of these systems. As a result, a great deal of research has gone into methods of more effectively learning from limited amounts of data.\n",
      "2000s: With the growth of the web, increasing amounts of raw (unannotated) language data has become available since the mid-1990s. Research has thus increasingly focused on unsupervised and semi-supervised learning algorithms.  Such algorithms can learn from data that has not been hand-annotated with the desired answers or using a combination of annotated and non-annotated data.  Generally, this task is much more difficult than supervised learning, and typically produces less accurate results for a given amount of input data.  However, there is an enormous amount of non-annotated data available (including, among other things, the entire content of the World Wide Web), which can often make up for the inferior results if the algorithm used has a low enough time complexity to be practical.\n",
      "Neural NLP (present)[edit]\n",
      "In the 2010s, representation learning and deep neural network-style machine learning methods became widespread in natural language processing. That popularity was due partly to a flurry of results showing that such techniques[8][9] can achieve state-of-the-art results in many natural language tasks, e.g., in language modeling[10] and parsing.[11][12] This is increasingly important in medicine and healthcare, where NLP helps analyze notes and text in electronic health records that would otherwise be inaccessible for study when seeking to improve care.[13]\n",
      "\n",
      "Methods: Rules, statistics, neural networks[edit]\n",
      "In the early days, many language-processing systems were designed by symbolic methods, i.e., the hand-coding of a set of rules, coupled with a dictionary lookup:[14][15] such as by writing grammars or devising heuristic rules for stemming.\n",
      "\n",
      "More recent systems based on machine-learning algorithms have many advantages over hand-produced rules: \n",
      "\n",
      "The learning procedures used during machine learning automatically focus on the most common cases, whereas when writing rules by hand it is often not at all obvious where the effort should be directed.\n",
      "Automatic learning procedures can make use of statistical inference algorithms to produce models that are robust to unfamiliar input (e.g. containing words or structures that have not been seen before) and to erroneous input (e.g. with misspelled words or words accidentally omitted). Generally, handling such input gracefully with handwritten rules, or, more generally, creating systems of handwritten rules that make soft decisions, is extremely difficult, error-prone and time-consuming.\n",
      "Systems based on automatically learning the rules can be made more accurate simply by supplying more input data. However, systems based on handwritten rules can only be made more accurate by increasing the complexity of the rules, which is a much more difficult task. In particular, there is a limit to the complexity of systems based on handwritten rules, beyond which the systems become more and more unmanageable. However, creating more data to input to machine-learning systems simply requires a corresponding increase in the number of man-hours worked, generally without significant increases in the complexity of the annotation process.\n",
      "Despite the popularity of machine learning in NLP research, symbolic methods are still (2020) commonly used:\n",
      "\n",
      "when the amount of training data is insufficient to successfully apply machine learning methods, e.g., for the machine translation of low-resource languages such as provided by the Apertium system,\n",
      "for preprocessing in NLP pipelines, e.g., tokenization, or\n",
      "for postprocessing and transforming the output of NLP pipelines, e.g., for knowledge extraction from syntactic parses.\n",
      "Statistical methods[edit]\n",
      "Since the so-called \"statistical revolution\"[16][17] in the late 1980s and mid-1990s, much natural language processing research has relied heavily on machine learning. The machine-learning paradigm calls instead for using statistical inference to automatically learn such rules through the analysis of large corpora (the plural form of corpus, is a set of documents, possibly with human or computer annotations) of typical real-world examples.\n",
      "\n",
      "Many different classes of machine-learning algorithms have been applied to natural-language-processing tasks. These algorithms take as input a large set of \"features\" that are generated from the input data. Increasingly, however, research has focused on statistical models, which make soft, probabilistic decisions based on attaching real-valued weights to each input feature (complex-valued embeddings,[18] and neural networks in general have also been proposed, for e.g. speech[19]). Such models have the advantage that they can express the relative certainty of many different possible answers rather than only one, producing more reliable results when such a model is included as a component of a larger system.\n",
      "\n",
      "Some of the earliest-used machine learning algorithms, such as decision trees, produced systems of hard if-then rules similar to existing hand-written rules.  However, part-of-speech tagging introduced the use of hidden Markov models to natural language processing, and increasingly, research has focused on statistical models, which make soft, probabilistic decisions based on attaching real-valued weights to the features making up the input data. The cache language models upon which many speech recognition systems now rely are examples of such statistical models.  Such models are generally more robust when given unfamiliar input, especially input that contains errors (as is very common for real-world data), and produce more reliable results when integrated into a larger system comprising multiple subtasks.\n",
      "\n",
      "Since the neural turn, statistical methods in NLP research have been largely replaced by neural networks. However, they continue to be relevant for contexts in which statistical interpretability and transparency is required.\n",
      "\n",
      "Neural networks[edit]\n",
      "A major drawback of statistical methods is that they require elaborate feature engineering. Since 2015,[20] the field has thus largely abandoned statistical methods and shifted to neural networks for machine learning. Popular techniques include the use of word embeddings to capture semantic properties of words, and an increase in end-to-end learning of a higher-level task (e.g., question answering) instead of relying on a pipeline of separate intermediate tasks (e.g., part-of-speech tagging and dependency parsing). In some areas, this shift has entailed substantial changes in how NLP systems are designed, such that deep neural network-based approaches may be viewed as a new paradigm distinct from statistical natural language processing. For instance, the term neural machine translation (NMT) emphasizes the fact that deep learning-based approaches to machine translation directly learn sequence-to-sequence transformations, obviating the need for intermediate steps such as word alignment and language modeling that was used in statistical machine translation (SMT).\n",
      "\n",
      "Common NLP tasks[edit]\n",
      "The following is a list of some of the most commonly researched tasks in natural language processing. Some of these tasks have direct real-world applications, while others more commonly serve as subtasks that are used to aid in solving larger tasks.\n",
      "\n",
      "Though natural language processing tasks are closely intertwined, they can be subdivided into categories for convenience. A coarse division is given below.\n",
      "\n",
      "Text and speech processing[edit]\n",
      "Morphological analysis[edit]\n",
      "Syntactic analysis[edit]\n",
      "Lexical semantics (of individual words in context)[edit]\n",
      "Relational semantics (semantics of individual sentences)[edit]\n",
      "Discourse (semantics beyond individual sentences)[edit]\n",
      "Higher-level NLP applications[edit]\n",
      "General tendencies and (possible) future directions[edit]\n",
      "Based on long-standing trends in the field, it is possible to extrapolate future directions of NLP. As of 2020, three trends among the topics of the long-standing series of CoNLL Shared Tasks can be observed:[41]\n",
      "\n",
      "Interest on increasingly abstract, \"cognitive\" aspects of natural language (1999–2001: shallow parsing, 2002–03: named entity recognition, 2006–09/2017–18: dependency syntax, 2004–05/2008–09 semantic role labelling, 2011–12 coreference, 2015–16: discourse parsing, 2019: semantic parsing).\n",
      "Increasing interest in multilinguality, and, potentially, multimodality (English since 1999; Spanish, Dutch since 2002; German since 2003; Bulgarian, Danish, Japanese, Portuguese, Slovenian, Swedish, Turkish since 2006; Basque, Catalan, Chinese, Greek, Hungarian, Italian, Turkish since 2007; Czech since 2009; Arabic since 2012; 2017: 40+ languages; 2018: 60+/100+ languages)\n",
      "Elimination of symbolic representations (rule-based over supervised towards weakly supervised methods, representation learning and end-to-end systems)\n",
      "Cognition and NLP[edit]\n",
      "Most higher-level NLP applications involve aspects that emulate intelligent behaviour and apparent comprehension of natural language. More broadly speaking, the technical operationalization of increasingly advanced aspects of cognitive behaviour represents one of the developmental trajectories of NLP (see trends among CoNLL shared tasks above).\n",
      "\n",
      "Cognition refers to \"the mental action or process of acquiring knowledge and understanding through thought, experience, and the senses.\"[42] Cognitive science is the interdisciplinary, scientific study of the mind and its processes.[43] Cognitive linguistics is an interdisciplinary branch of linguistics, combining knowledge and research from both psychology and linguistics.[44] Especially during the age of symbolic NLP, the area of computational linguistics maintained strong ties with cognitive studies.\n",
      "\n",
      "As an example, George Lakoff offers a methodology to build natural language processing (NLP) algorithms through the perspective of cognitive science, along with the findings of cognitive linguistics,[45] with two defining aspects:\n",
      "\n",
      "Ties with cognitive linguistics are part of the historical heritage of NLP, but they have been less frequently addressed since the statistical turn during the 1990s. Nevertheless, approaches to develop cognitive models towards technically operationalizable frameworks have been pursued in the context of various frameworks, e.g., of cognitive grammar,[47] functional grammar,[48] construction grammar,[49] computational psycholinguistics and cognitive neuroscience (e.g., ACT-R), however, with limited uptake in mainstream NLP (as measured by presence on major conferences[50] of the ACL). More recently, ideas of cognitive NLP have been revived as an approach to achieve explainability, e.g., under the notion of \"cognitive AI\".[51] Likewise, ideas of cognitive NLP are inherent to neural models multimodal NLP (although rarely made explicit).[52]\n",
      "\n",
      "See also[edit]\n",
      "1 the Road\n",
      "Automated essay scoring\n",
      "Biomedical text mining\n",
      "Compound term processing\n",
      "Computational linguistics\n",
      "Computer-assisted reviewing\n",
      "Controlled natural language\n",
      "Deep learning\n",
      "Deep linguistic processing\n",
      "Distributional semantics\n",
      "Foreign language reading aid\n",
      "Foreign language writing aid\n",
      "Information extraction\n",
      "Information retrieval\n",
      "Language and Communication Technologies\n",
      "Language model\n",
      "Language technology\n",
      "Latent semantic indexing\n",
      "Multi-agent system\n",
      "Native-language identification\n",
      "Natural-language programming\n",
      "Natural-language understanding\n",
      "Natural-language search\n",
      "Outline of natural language processing\n",
      "Query expansion\n",
      "Query understanding\n",
      "Reification (linguistics)\n",
      "Speech processing\n",
      "Spoken dialogue systems\n",
      "Text-proofing\n",
      "Text simplification\n",
      "Transformer (machine learning model)\n",
      "Truecasing\n",
      "Question answering\n",
      "Word2vec\n",
      "References[edit]\n",
      "Further reading[edit]\n",
      "Bates, M (1995). \"Models of natural language understanding\". Proceedings of the National Academy of Sciences of the United States of America. 92 (22): 9977–9982. Bibcode:1995PNAS...92.9977B. doi:10.1073/pnas.92.22.9977. PMC 40721. PMID 7479812.\n",
      "Steven Bird, Ewan Klein, and Edward Loper (2009). Natural Language Processing with Python. O'Reilly Media. ISBN 978-0-596-51649-9.\n",
      "Daniel Jurafsky and James H. Martin (2008). Speech and Language Processing, 2nd edition. Pearson Prentice Hall. ISBN 978-0-13-187321-6.\n",
      "Mohamed Zakaria Kurdi (2016). Natural Language Processing and Computational Linguistics: speech, morphology, and syntax, Volume 1. ISTE-Wiley. ISBN 978-1848218482.\n",
      "Mohamed Zakaria Kurdi (2017). Natural Language Processing and Computational Linguistics: semantics, discourse, and applications, Volume 2. ISTE-Wiley. ISBN 978-1848219212.\n",
      "Christopher D. Manning, Prabhakar Raghavan, and Hinrich Schütze (2008). Introduction to Information Retrieval. Cambridge University Press. ISBN 978-0-521-86571-5. Official html and pdf versions available without charge.\n",
      "Christopher D. Manning and Hinrich Schütze (1999). Foundations of Statistical Natural Language Processing. The MIT Press. ISBN 978-0-262-13360-9.\n",
      "David M. W. Powers and Christopher C. R. Turk (1989). Machine Learning of Natural Language. Springer-Verlag. ISBN 978-0-387-19557-5.\n",
      "External links[edit]\n",
      " Media related to Natural language processing at Wikimedia Commons\n",
      "vte\n",
      "AI-complete\n",
      "Bag-of-words\n",
      "n-gram\n",
      "Bigram\n",
      "Trigram\n",
      "Computational linguistics\n",
      "Natural-language understanding\n",
      "Stop words\n",
      "Text processing\n",
      "Bigram\n",
      "Trigram\n",
      "Collocation extraction\n",
      "Concept mining\n",
      "Coreference resolution\n",
      "Deep linguistic processing\n",
      "Distant reading\n",
      "Information extraction\n",
      "Named-entity recognition\n",
      "Ontology learning\n",
      "Parsing\n",
      "Part-of-speech tagging\n",
      "Semantic role labeling\n",
      "Semantic similarity\n",
      "Sentiment analysis\n",
      "Terminology extraction\n",
      "Text mining\n",
      "Textual entailment\n",
      "Truecasing\n",
      "Word-sense disambiguation\n",
      "Word-sense induction\n",
      "Compound-term processing\n",
      "Lemmatisation\n",
      "Lexical analysis\n",
      "Text chunking\n",
      "Stemming\n",
      "Sentence segmentation\n",
      "Word segmentation\n",
      "Multi-document summarization\n",
      "Sentence extraction\n",
      "Text simplification\n",
      "Computer-assisted\n",
      "Example-based\n",
      "Rule-based\n",
      "Statistical\n",
      "Transfer-based\n",
      "Neural\n",
      "BERT\n",
      "Document-term matrix\n",
      "Explicit semantic analysis\n",
      "fastText\n",
      "GloVe\n",
      "Language model\n",
      "Latent semantic analysis\n",
      "Seq2seq\n",
      "Word embedding\n",
      "Word2vec\n",
      "Corpus linguistics\n",
      "Lexical resource\n",
      "Linguistic Linked Open Data\n",
      "Machine-readable dictionary\n",
      "Parallel text\n",
      "PropBank\n",
      "Semantic network\n",
      "Simple Knowledge Organization System\n",
      "Speech corpus\n",
      "Text corpus\n",
      "Thesaurus (information retrieval)\n",
      "Treebank\n",
      "Universal Dependencies\n",
      "BabelNet\n",
      "Bank of English\n",
      "DBpedia\n",
      "FrameNet\n",
      "Google Ngram Viewer\n",
      "UBY\n",
      "WordNet\n",
      "Speech recognition\n",
      "Speech segmentation\n",
      "Speech synthesis\n",
      "Natural language generation\n",
      "Optical character recognition\n",
      "Document classification\n",
      "Latent Dirichlet allocation\n",
      "Pachinko allocation\n",
      "Automated essay scoring\n",
      "Concordancer\n",
      "Grammar checker\n",
      "Predictive text\n",
      "Pronunciation assessment\n",
      "Spell checker\n",
      "Syntax guessing\n",
      "Chatbot\n",
      "Interactive fiction\n",
      "Question answering\n",
      "Virtual assistant\n",
      "Voice user interface\n",
      "Hallucination\n",
      "Natural Language Toolkit\n",
      "spaCy\n",
      " Language\n",
      "Israel\n",
      "United States\n",
      "Japan\n",
      "Czech Republic\n",
      "Natural language processingComputational fields of studyComputational linguisticsSpeech recognition\n",
      "All accuracy disputesAccuracy disputes from December 2013CS1 maint: locationArticles with short descriptionShort description is different from WikidataCommons category link from WikidataArticles with J9U identifiersArticles with LCCN identifiersArticles with NDL identifiersArticles with NKC identifiers\n",
      "\n",
      " This page was last edited on 25 February 2023, at 00:09 (UTC).\n",
      "Text is available under the Creative Commons Attribution-ShareAlike License 3.0;\n",
      "additional terms may apply.  By using this site, you agree to the Terms of Use and Privacy Policy. Wikipedia® is a registered trademark of the Wikimedia Foundation, Inc., a non-profit organization.\n",
      "\n",
      "\n",
      "Privacy policy\n",
      "About Wikipedia\n",
      "Disclaimers\n",
      "Contact Wikipedia\n",
      "Mobile view\n",
      "Developers\n",
      "Statistics\n",
      "Cookie statement\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# define the website URL to extract text from\n",
    "url = \"https://en.wikipedia.org/wiki/Natural_language_processing\"\n",
    "\n",
    "# send a request to the website and get the HTML content\n",
    "response = requests.get(url)\n",
    "html_content = response.text\n",
    "\n",
    "# create a Beautiful Soup object and parse the HTML content\n",
    "soup = BeautifulSoup(html_content, 'html.parser')\n",
    "\n",
    "# find all the <p> elements on the page\n",
    "paragraphs=soup.find_all(['p', 'h1', 'h2', 'h3','ul'])\n",
    "\n",
    "# extract the text content from each <p> element\n",
    "for paragraph in paragraphs:\n",
    "    text = paragraph.get_text()\n",
    "    print(text)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# protoype\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from googlesearch import search\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import tkinter as tk\n",
    "from PIL import Image, ImageTk\n",
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt\n",
    "text=\"\"\n",
    "\n",
    "def get_text_from_url(url):\n",
    "    # Specify url of the web page\n",
    "    source = urlopen(url).read()\n",
    "\n",
    "    # Make a soup\n",
    "    soup = BeautifulSoup(source, 'lxml')\n",
    "\n",
    "    # Extract the plain text content from paragraphs\n",
    "    paras = []\n",
    "    for paragraph in soup.find_all('p'):\n",
    "        paras.append(str(paragraph.text))\n",
    "\n",
    "    # Extract text from paragraph headers\n",
    "    heads = []\n",
    "    for head in soup.find_all('span', attrs={'mw-headline'}):\n",
    "        heads.append(str(head.text))\n",
    "\n",
    "    # Interleave paragraphs & headers\n",
    "    text = [val for pair in zip(paras, heads) for val in pair]\n",
    "    text = ' '.join(text)\n",
    "\n",
    "    # Drop footnote superscripts in brackets\n",
    "    text = re.sub(r\"\\[.*?\\]+\", '', text)\n",
    "\n",
    "    # Replace '\\n' (a new line) with '' and end the string at $1000.\n",
    "    text = text.replace('\\n', '')[:-11]\n",
    "    \n",
    "    return text\n",
    "\n",
    "\n",
    "def generate_and_show_wordcloud():\n",
    "    text = output_text.get(\"1.0\", tk.END)\n",
    "    if text.strip() == \"\":\n",
    "        return\n",
    "    else:\n",
    "        # text = ' '.join([word for word in text.split() if word.lower() not in stop_words])\n",
    "        wordcloud = WordCloud(width=400, height=400, background_color=\"white\").generate(text)\n",
    "        plt.imshow(wordcloud, interpolation='bilinear')\n",
    "        plt.axis(\"off\")\n",
    "        plt.tight_layout(pad=0)\n",
    "        plt.savefig(\"wordcloud.png\")\n",
    "        plt.close()\n",
    "\n",
    "        image = Image.open(\"wordcloud.png\")\n",
    "        image = image.convert(\"RGB\")\n",
    "        img = ImageTk.PhotoImage(image)\n",
    "        canvas.create_image(0, 0, anchor=tk.NW, image=img)\n",
    "        canvas.image = img\n",
    "\n",
    "def search_and_display(event=None):\n",
    "    query = search_box.get()\n",
    "    url = None\n",
    "    for j in search(query, tld=\"co.in\", num=1, stop=2, pause=5):\n",
    "        url = j\n",
    "        time.sleep(5)\n",
    "    if url:\n",
    "        text = get_text_from_url(url)\n",
    "        output_text.delete('1.0', tk.END)\n",
    "        output_text.insert(tk.END, text)\n",
    "       \n",
    "    else:\n",
    "        output_text.delete('1.0', tk.END)\n",
    "        output_text.insert(tk.END, \"No results found.\")\n",
    "\n",
    "window = tk.Tk()\n",
    "window.title(\"Search bot\")\n",
    "\n",
    "search_label = tk.Label(window, text=\"Enter search query:\")\n",
    "search_label.pack()\n",
    "search_box = tk.Entry(window)\n",
    "search_box.pack()\n",
    "submit_button = tk.Button(window, text=\"Submit\", command=search_and_display)\n",
    "submit_button.pack()\n",
    "\n",
    "word_button = tk.Button(window, text=\"word cloud\", command=generate_and_show_wordcloud)\n",
    "word_button.pack()\n",
    "\n",
    "canvas = tk.Canvas(window, width=400, height=400)\n",
    "canvas.pack()\n",
    "\n",
    "output_label = tk.Label(window, text=\"Website content:\")\n",
    "output_label.pack()\n",
    "output_text = tk.Text(window, height=10)\n",
    "output_text.pack()\n",
    "\n",
    "window.mainloop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from googlesearch import search\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import tkinter as tk\n",
    "from PIL import Image, ImageTk\n",
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt\n",
    "import gensim.summarization\n",
    "import time\n",
    "\n",
    "# Function to extract plain text content from a URL\n",
    "def get_text_from_url(url):\n",
    "    # Specify url of the web page\n",
    "    source = urlopen(url).read()\n",
    "\n",
    "    # Make a soup\n",
    "    soup = BeautifulSoup(source, 'lxml')\n",
    "\n",
    "    # Extract the plain text content from paragraphs\n",
    "    paras = []\n",
    "    for paragraph in soup.find_all('p'):\n",
    "        paras.append(str(paragraph.text))\n",
    "\n",
    "    # Extract text from paragraph headers\n",
    "    heads = []\n",
    "    for head in soup.find_all('span', attrs={'mw-headline'}):\n",
    "        heads.append(str(head.text))\n",
    "\n",
    "    # Interleave paragraphs & headers\n",
    "    text = [val for pair in zip(paras, heads) for val in pair]\n",
    "    text = ' '.join(text)\n",
    "\n",
    "    # Drop footnote superscripts in brackets\n",
    "    text = re.sub(r\"\\[.*?\\]+\", '', text)\n",
    "\n",
    "    # Replace '\\n' (a new line) with '' and end the string at $1000.\n",
    "    text = text.replace('\\n', '')[:-11]\n",
    "\n",
    "    return text\n",
    "\n",
    "# Function to generate and display word cloud\n",
    "def generate_and_show_wordcloud():\n",
    "    text = output_text.get(\"1.0\", tk.END)\n",
    "    if text.strip() == \"\":\n",
    "        return\n",
    "    else:\n",
    "        # Generate word cloud\n",
    "        wordcloud = WordCloud(width=400, height=400, background_color=\"white\").generate(text)\n",
    "\n",
    "        # Save word cloud as image\n",
    "        plt.imshow(wordcloud, interpolation='bilinear')\n",
    "        plt.axis(\"off\")\n",
    "        plt.tight_layout(pad=0)\n",
    "        plt.savefig(\"wordcloud.png\")\n",
    "        plt.close()\n",
    "\n",
    "        # Display word cloud on canvas\n",
    "        image = Image.open(\"wordcloud.png\")\n",
    "        image = image.convert(\"RGB\")\n",
    "        img = ImageTk.PhotoImage(image)\n",
    "        canvas.create_image(0, 0, anchor=tk.NW, image=img)\n",
    "        canvas.image = img\n",
    "\n",
    "# Function to search for query and display website content\n",
    "def search_and_display(event=None):\n",
    "    query = search_box.get()\n",
    "    url = None\n",
    "\n",
    "    # Perform Google search and extract URL of first result\n",
    "    for j in search(query, tld=\"co.in\", num=1, stop=2, pause=5):\n",
    "        url = j\n",
    "        time.sleep(5)\n",
    "\n",
    "    # If URL found, extract text and generate summary\n",
    "    if url:\n",
    "        text = get_text_from_url(url)\n",
    "        summary = gensim.summarization.summarize(text)\n",
    "        output_text.delete('1.0', tk.END)\n",
    "        output_text.insert(tk.END, summary)\n",
    "    else:\n",
    "        output_text.delete('1.0', tk.END)\n",
    "        output_text.insert(tk.END, \"No results found.\")\n",
    "\n",
    "# Create window\n",
    "window = tk.Tk()\n",
    "window.title(\"Search bot\")\n",
    "window.geometry('600x700')\n",
    "\n",
    "# Search box\n",
    "search_label = tk.Label(window, text=\"Enter search query:\",font=('Calibri',14))\n",
    "search_label.pack(pady=(10,0))\n",
    "search_box = tk.Entry(window, font=('Calibri', 14))\n",
    "search_box.pack(pady=(0,10))\n",
    "\n",
    "# Submit button\n",
    "submit_button = tk.Button(window, text=\"Submit\", command=search_and_display)\n",
    "submit_button.pack(pady=(0,10))\n",
    "\n",
    "# Word cloud button and canvas\n",
    "word_button = tk.Button(window, text=\"Generate Word Cloud\", command=generate_and_show_wordcloud)\n",
    "word_button.pack()\n",
    "\n",
    "canvas = tk.Canvas(window, width=400, height=400)\n",
    "canvas.pack( padx=20)\n",
    "\n",
    "output_label = tk.Label(window, text=\"Website content:\",font=('Calibri',14))\n",
    "output_label.pack()\n",
    "output_text = tk.Text(window, height=10)\n",
    "output_text.pack()\n",
    "\n",
    "window.mainloop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jithi\\AppData\\Local\\Temp\\ipykernel_24024\\3168583570.py:90: DeprecationWarning: ANTIALIAS is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.LANCZOS instead.\n",
      "  bg_image = bg_image.resize((600, 900), Image.ANTIALIAS)\n"
     ]
    }
   ],
   "source": [
    "from googlesearch import search\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import tkinter as tk\n",
    "from PIL import Image, ImageTk\n",
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt\n",
    "import gensim.summarization\n",
    "import time\n",
    "\n",
    "# Function to extract plain text content from a URL\n",
    "def get_text_from_url(url):\n",
    "    # Specify url of the web page\n",
    "    source = urlopen(url).read()\n",
    "\n",
    "    # Make a soup\n",
    "    soup = BeautifulSoup(source, 'lxml')\n",
    "\n",
    "    # Extract the plain text content from paragraphs\n",
    "    paras = []\n",
    "    for paragraph in soup.find_all('p'):\n",
    "        paras.append(str(paragraph.text))\n",
    "\n",
    "    # Extract text from paragraph headers\n",
    "    heads = []\n",
    "    for head in soup.find_all('span', attrs={'mw-headline'}):\n",
    "        heads.append(str(head.text))\n",
    "\n",
    "    # Interleave paragraphs & headers\n",
    "    text = [val for pair in zip(paras, heads) for val in pair]\n",
    "    text = ' '.join(text)\n",
    "\n",
    "    # Drop footnote superscripts in brackets\n",
    "    text = re.sub(r\"\\[.*?\\]+\", '', text)\n",
    "\n",
    "    # Replace '\\n' (a new line) with '' and end the string at $1000.\n",
    "    text = text.replace('\\n', '')[:-11]\n",
    "\n",
    "    return text\n",
    "\n",
    "# Function to generate and display word cloud\n",
    "def generate_and_show_wordcloud():\n",
    "    text = output_text.get(\"1.0\", tk.END)\n",
    "    if text.strip() == \"\":\n",
    "        return\n",
    "    else:\n",
    "        # Generate word cloud\n",
    "        wordcloud = WordCloud(width=600, height=600, background_color=\"white\").generate(text)\n",
    "\n",
    "        # Save word cloud as image\n",
    "        plt.imshow(wordcloud, interpolation='bilinear')\n",
    "        plt.axis(\"off\")\n",
    "        plt.tight_layout(pad=0)\n",
    "        plt.savefig(\"wordcloud.png\")\n",
    "        plt.close()\n",
    "\n",
    "        # Display word cloud on canvas\n",
    "        image = Image.open(\"wordcloud.png\")\n",
    "        image = image.convert(\"RGB\")\n",
    "        img = ImageTk.PhotoImage(image)\n",
    "        canvas.create_image(0, 0, anchor=tk.NW, image=img)\n",
    "        canvas.image = img\n",
    "\n",
    "# Function to search for query and display website content\n",
    "def search_and_display(event=None):\n",
    "    query = search_box.get()\n",
    "    url = None\n",
    "\n",
    "    # Perform Google search and extract URL of first result\n",
    "    for j in search(query, tld=\"co.in\", num=1, stop=2, pause=5):\n",
    "        url = j\n",
    "        time.sleep(5)\n",
    "\n",
    "    # If URL found, extract text and generate summary\n",
    "    if url:\n",
    "        text = get_text_from_url(url)\n",
    "        summary = gensim.summarization.summarize(text,ratio=0.8)\n",
    "        output_text.delete('1.0', tk.END)\n",
    "        output_text.insert(tk.END, summary)\n",
    "    else:\n",
    "        output_text.delete('1.0', tk.END)\n",
    "        output_text.insert(tk.END, \"No results found.\")\n",
    "\n",
    "# Create window\n",
    "window = tk.Tk()\n",
    "window.title(\"Search bot\")\n",
    "window.geometry('600x900')\n",
    "bg_image = Image.open(\"C:\\\\Users\\\\jithi\\\\OneDrive\\\\Desktop\\\\bg2.png\")\n",
    "bg_image = bg_image.resize((600, 900), Image.ANTIALIAS)\n",
    "\n",
    "image = ImageTk.PhotoImage(bg_image)\n",
    "background_label = tk.Label(window, image=image)\n",
    "background_label.place(x=0, y=0, relwidth=1, relheight=1)\n",
    "\n",
    "\n",
    "\n",
    "# Set the background of the window to the label\n",
    "window.configure()\n",
    "# Search box\n",
    "search_label = tk.Label(window, text=\"Enter search query:\",font=('Calibri',14))\n",
    "search_label.pack(pady=(10,0))\n",
    "search_box = tk.Entry(window, font=('Calibri', 14))\n",
    "search_box.pack(pady=(0,10))\n",
    "\n",
    "# Submit button\n",
    "submit_button = tk.Button(window, text=\"Submit\", command=search_and_display)\n",
    "submit_button.pack(pady=(0,10))\n",
    "\n",
    "# Word cloud button and canvas\n",
    "word_button = tk.Button(window, text=\"Generate Word Cloud\", command=generate_and_show_wordcloud)\n",
    "word_button.pack()\n",
    "\n",
    "canvas = tk.Canvas(window, width=400, height=400)\n",
    "canvas.pack()\n",
    "\n",
    "output_label = tk.Label(window, text=\"Website content:\",font=('Calibri',14))\n",
    "output_label.pack()\n",
    "output_text = tk.Text(window, height=10)\n",
    "output_text.pack()\n",
    "\n",
    "window.mainloop()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from googlesearch import search\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import tkinter as tk\n",
    "from PIL import Image, ImageTk\n",
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt\n",
    "import gensim.summarization\n",
    "import time\n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "def google_search(query):\n",
    "    url = f\"https://www.google.com/search?q={query}\"\n",
    "    headers = {\n",
    "        \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36\"}\n",
    "    response = requests.get(url, headers=headers)\n",
    "    soup = BeautifulSoup(response.text, \"html.parser\")\n",
    "    search_results = soup.find_all(\"div\", class_=\"g\")\n",
    "    first_result = search_results[0].find(\"a\")[\"href\"]\n",
    "    return first_result\n",
    "\n",
    "# Function to extract plain text content from a URL\n",
    "def get_text_from_url(url):\n",
    "    # Specify url of the web page\n",
    "    source = urlopen(url).read()\n",
    "\n",
    "    # Make a soup\n",
    "    soup = BeautifulSoup(source, 'lxml')\n",
    "    # Extract the plain text content from paragraphs\n",
    "    paras = []\n",
    "    for paragraph in soup.find_all('p'):\n",
    "        paras.append(str(paragraph.text))\n",
    "\n",
    "    # Extract text from paragraph headers\n",
    "    heads = []\n",
    "    for head in soup.find_all('span', attrs={'mw-headline'}):\n",
    "        heads.append(str(head.text))\n",
    "\n",
    "    # Interleave paragraphs & headers\n",
    "    text = [val for pair in zip(paras, heads) for val in pair]\n",
    "    text = ' '.join(text)\n",
    "    text = re.sub(r\"\\[.*?\\]+\", '', text)\n",
    "    text = text.replace('\\n', '')[:-11]\n",
    "\n",
    "    return text\n",
    "\n",
    "# Function to generate and display word cloud\n",
    "def generate_and_show_wordcloud():\n",
    "    text = output_text.get(\"1.0\", tk.END)\n",
    "    if text.strip() == \"\":\n",
    "        return\n",
    "    else:\n",
    "        # Generate word cloud\n",
    "        wordcloud = WordCloud(width=600, height=600, background_color=\"white\").generate(text)\n",
    "\n",
    "        # Save word cloud as image\n",
    "        plt.imshow(wordcloud, interpolation='bilinear')\n",
    "        plt.axis(\"off\")\n",
    "        plt.tight_layout(pad=0)\n",
    "        plt.savefig(\"wordcloud.png\")\n",
    "        plt.close()\n",
    "\n",
    "        # Display word cloud on canvas\n",
    "        image = Image.open(\"wordcloud.png\")\n",
    "        image = image.convert(\"RGB\")\n",
    "        img = ImageTk.PhotoImage(image)\n",
    "        canvas.create_image(0, 0, anchor=tk.NW, image=img)\n",
    "        canvas.image = img\n",
    "\n",
    "# Function to search for query and display website content\n",
    "def search_and_display(event=None):\n",
    "    query = search_box.get()\n",
    "    url = None\n",
    "     \n",
    "    # Perform Google search and extract URL of first result\n",
    "    url=google_search(query)\n",
    "\n",
    "    # If URL found, extract text and generate summary\n",
    "    if url:\n",
    "        text = get_text_from_url(url)\n",
    "        summary = gensim.summarization.summarize(text)\n",
    "        output_text.delete('1.0', tk.END)\n",
    "        output_text.insert(tk.END, summary)\n",
    "    else:\n",
    "        output_text.delete('1.0', tk.END)\n",
    "        output_text.insert(tk.END, \"No results found.\")\n",
    "\n",
    "# Create window\n",
    "window = tk.Tk()\n",
    "window.title(\"Search bot\")\n",
    "window.geometry('600x900')\n",
    "bg_image = Image.open(\"C:\\\\Users\\\\jithi\\\\OneDrive\\\\Desktop\\\\bg2.png\")\n",
    "bg_image = bg_image.resize((600, 900), Image.Resampling.LANCZOS)\n",
    "\n",
    "image = ImageTk.PhotoImage(bg_image)\n",
    "background_label = tk.Label(window, image=image)\n",
    "background_label.place(x=0, y=0, relwidth=1, relheight=1)\n",
    "\n",
    "\n",
    "\n",
    "# Set the background of the window to the label\n",
    "window.configure()\n",
    "# Search box\n",
    "search_label = tk.Label(window, text=\"Enter search query:\",font=('Calibri',14))\n",
    "search_label.pack(pady=(10,0))\n",
    "search_box = tk.Entry(window, font=('Calibri', 14))\n",
    "search_box.pack(pady=(0,10))\n",
    "\n",
    "# Submit button\n",
    "submit_button = tk.Button(window, text=\"Submit\", command=search_and_display)\n",
    "submit_button.pack(pady=(0,10))\n",
    "\n",
    "# Word cloud button and canvas\n",
    "word_button = tk.Button(window, text=\"Generate Word Cloud\", command=generate_and_show_wordcloud)\n",
    "word_button.pack()\n",
    "\n",
    "canvas = tk.Canvas(window, width=400, height=400)\n",
    "canvas.pack()\n",
    "\n",
    "output_label = tk.Label(window, text=\"Website content:\",font=('Calibri',14))\n",
    "output_label.pack()\n",
    "output_text = tk.Text(window, height=10)\n",
    "output_text.pack()\n",
    "\n",
    "window.mainloop()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "time_taken_to_get_url=0\n",
    "from googlesearch import search\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import tkinter as tk\n",
    "from PIL import Image, ImageTk\n",
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt\n",
    "import gensim.summarization\n",
    "import time\n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "def google_search(query):\n",
    "    url = f\"https://www.google.com/search?q={query}\"\n",
    "    headers = {\n",
    "        \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36\"}\n",
    "    response = requests.get(url, headers=headers)\n",
    "    soup = BeautifulSoup(response.text, \"html.parser\")\n",
    "    search_results = soup.find_all(\"div\", class_=\"g\")\n",
    "    first_result = search_results[0].find(\"a\")[\"href\"]\n",
    "    return first_result\n",
    "\n",
    "# Function to extract plain text content from a URL\n",
    "def get_text_from_url(url):\n",
    "    # Specify url of the web page\n",
    "    source = urlopen(url).read()\n",
    "\n",
    "    # Make a soup\n",
    "    soup = BeautifulSoup(source, 'lxml')\n",
    "    # Extract the plain text content from paragraphs\n",
    "    paras = []\n",
    "    for paragraph in soup.find_all('p'):\n",
    "        paras.append(str(paragraph.text))\n",
    "\n",
    "    # Extract text from paragraph headers\n",
    "    heads = []\n",
    "    for head in soup.find_all('span', attrs={'mw-headline'}):\n",
    "        heads.append(str(head.text))\n",
    "\n",
    "    # Interleave paragraphs & headers\n",
    "    text = [val for pair in zip(paras, heads) for val in pair]\n",
    "    text = ' '.join(text)\n",
    "    text = re.sub(r\"\\[.*?\\]+\", '', text)\n",
    "    text = text.replace('\\n', '')[:-11]\n",
    "\n",
    "    return text\n",
    "\n",
    "# Function to generate and display word cloud\n",
    "def generate_and_show_wordcloud():\n",
    "    text = output_text.get(\"1.0\", tk.END)\n",
    "    if text.strip() == \"\":\n",
    "        return\n",
    "    else:\n",
    "        # Generate word cloud\n",
    "        wordcloud = WordCloud(width=600, height=600, background_color=\"white\").generate(text)\n",
    "\n",
    "        # Save word cloud as image\n",
    "        plt.imshow(wordcloud, interpolation='bilinear')\n",
    "        plt.axis(\"off\")\n",
    "        plt.tight_layout(pad=0)\n",
    "        plt.savefig(\"wordcloud.png\")\n",
    "        plt.close()\n",
    "\n",
    "        # Display word cloud on canvas\n",
    "        image = Image.open(\"wordcloud.png\")\n",
    "        image = image.convert(\"RGB\")\n",
    "        img = ImageTk.PhotoImage(image)\n",
    "        canvas.create_image(0, 0, anchor=tk.NW, image=img)\n",
    "        canvas.image = img\n",
    "\n",
    "# Function to search for query and display website content\n",
    "def search_and_display(event=None):\n",
    "    query = search_box.get()\n",
    "    url = None\n",
    "\n",
    "    # Perform Google search and extract URL of first result\n",
    "    start_time = time.time()\n",
    "    url = google_search(query)\n",
    "    end_time = time.time()\n",
    "    time_taken_to_get_url = end_time - start_time\n",
    "\n",
    "    # If URL found, extract text and generate summary\n",
    "    if url:\n",
    "        start_time = time.time()\n",
    "        text = get_text_from_url(url)\n",
    "        summary = gensim.summarization.summarize(text)\n",
    "        end_time = time.time()\n",
    "        output_text.delete('1.0', tk.END)\n",
    "        output_text.insert(tk.END, summary)\n",
    "        time_taken_to_summarize_text = end_time - start_time\n",
    "\n",
    "        # Calculate percentage of words removed in summarization\n",
    "        original_word_count = len(text.split())\n",
    "        summarized_word_count = len(summary.split())\n",
    "        percent_words_removed = (original_word_count - summarized_word_count) / original_word_count * 100\n",
    "        return percent_words_removed,time_taken_to_get_url\n",
    "# Create window\n",
    "window = tk.Tk()\n",
    "window.title(\"Search bot\")\n",
    "window.geometry('600x900')\n",
    "bg_image = Image.open(\"C:\\\\Users\\\\jithi\\\\OneDrive\\\\Desktop\\\\bg2.png\")\n",
    "bg_image = bg_image.resize((600, 900), Image.Resampling.LANCZOS)\n",
    "\n",
    "image = ImageTk.PhotoImage(bg_image)\n",
    "background_label = tk.Label(window, image=image)\n",
    "background_label.place(x=0, y=0, relwidth=1, relheight=1)\n",
    "\n",
    "\n",
    "\n",
    "# Set the background of the window to the label\n",
    "window.configure()\n",
    "# Search box\n",
    "search_label = tk.Label(window, text=\"Enter search query:\",font=('Calibri',14))\n",
    "search_label.pack(pady=(10,0))\n",
    "search_box = tk.Entry(window, font=('Calibri', 14))\n",
    "search_box.pack(pady=(0,10))\n",
    "\n",
    "# Submit button\n",
    "submit_button = tk.Button(window, text=\"Submit\", command=search_and_display)\n",
    "submit_button.pack(pady=(0,10))\n",
    "\n",
    "# Word cloud button and canvas\n",
    "word_button = tk.Button(window, text=\"Generate Word Cloud\", command=generate_and_show_wordcloud)\n",
    "word_button.pack()\n",
    "\n",
    "canvas = tk.Canvas(window, width=400, height=400)\n",
    "canvas.pack()\n",
    "\n",
    "output_label = tk.Label(window, text=\"Website content:\",font=('Calibri',14))\n",
    "output_label.pack()\n",
    "output_text = tk.Text(window, height=10)\n",
    "output_text.pack()\n",
    "\n",
    "window.mainloop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time_taken_to_get_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'percent_words_removed' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\jithi\\OneDrive\\Desktop\\VsCode\\labs\\nlp_labs\\nlp_project.ipynb Cell 23\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/jithi/OneDrive/Desktop/VsCode/labs/nlp_labs/nlp_project.ipynb#X31sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m percent_words_removed\n",
      "\u001b[1;31mNameError\u001b[0m: name 'percent_words_removed' is not defined"
     ]
    }
   ],
   "source": [
    "def google_search(query):\n",
    "    url = f\"https://www.google.com/search?q={query}\"\n",
    "    headers = {\n",
    "        \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36\"}\n",
    "    response = requests.get(url, headers=headers)\n",
    "    soup = BeautifulSoup(response.text, \"html.parser\")\n",
    "    search_results = soup.find_all(\"div\", class_=\"g\")\n",
    "    first_result = search_results[0].find(\"a\")[\"href\"]\n",
    "    return first_result\n",
    "\n",
    "# Function to extract plain text content from a URL\n",
    "def get_text_from_url(url):\n",
    "    # Specify url of the web page\n",
    "    source = urlopen(url).read()\n",
    "\n",
    "    # Make a soup\n",
    "    soup = BeautifulSoup(source, 'lxml')\n",
    "    # Extract the plain text content from paragraphs\n",
    "    paras = []\n",
    "    for paragraph in soup.find_all('p'):\n",
    "        paras.append(str(paragraph.text))\n",
    "\n",
    "    # Extract text from paragraph headers\n",
    "    heads = []\n",
    "    for head in soup.find_all('span', attrs={'mw-headline'}):\n",
    "        heads.append(str(head.text))\n",
    "\n",
    "    # Interleave paragraphs & headers\n",
    "    text = [val for pair in zip(paras, heads) for val in pair]\n",
    "    text = ' '.join(text)\n",
    "    text = re.sub(r\"\\[.*?\\]+\", '', text)\n",
    "    text = text.replace('\\n', '')[:-11]\n",
    "\n",
    "    return text\n",
    "\n",
    "# Function to generate and display word cloud\n",
    "def generate_and_show_wordcloud():\n",
    "    text = output_text.get(\"1.0\", tk.END)\n",
    "    if text.strip() == \"\":\n",
    "        return\n",
    "    else:\n",
    "        # Generate word cloud\n",
    "        wordcloud = WordCloud(width=600, height=600, background_color=\"white\").generate(text)\n",
    "\n",
    "        # Save word cloud as image\n",
    "        plt.imshow(wordcloud, interpolation='bilinear')\n",
    "        plt.axis(\"off\")\n",
    "        plt.tight_layout(pad=0)\n",
    "        plt.savefig(\"wordcloud.png\")\n",
    "        plt.close()\n",
    "\n",
    "        # Display word cloud on canvas\n",
    "        image = Image.open(\"wordcloud.png\")\n",
    "        image = image.convert(\"RGB\")\n",
    "        img = ImageTk.PhotoImage(image)\n",
    "        canvas.create_image(0, 0, anchor=tk.NW, image=img)\n",
    "        canvas.image = img\n",
    "\n",
    "# Function to search for query and display website content\n",
    "def search_and_display(event=None):\n",
    "    query = search_box.get()\n",
    "    url = None\n",
    "\n",
    "    # Perform Google search and extract URL of first result\n",
    "    start_time = time.time()\n",
    "    url = google_search(query)\n",
    "    end_time = time.time()\n",
    "    time_taken_to_get_url = end_time - start_time\n",
    "\n",
    "    # If URL found, extract text and generate summary\n",
    "    if url:\n",
    "        start_time = time.time()\n",
    "        text = get_text_from_url(url)\n",
    "        summary = gensim.summarization.summarize(text)\n",
    "        end_time = time.time()\n",
    "        output_text.delete('1.0', tk.END)\n",
    "        output_text.insert(tk.END, summary)\n",
    "        time_taken_to_summarize_text = end_time - start_time\n",
    "\n",
    "        # Calculate percentage of words removed in summarization\n",
    "        original_word_count = len(text.split())\n",
    "        summarized_word_count = len(summary.split())\n",
    "        percent_words_removed = (original_word_count - summarized_word_count) / original_word_count * 100\n",
    "        return percent_words_removed,time_taken_to_get_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'text' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\jithi\\OneDrive\\Desktop\\VsCode\\labs\\nlp_labs\\nlp_project.ipynb Cell 25\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/jithi/OneDrive/Desktop/VsCode/labs/nlp_labs/nlp_project.ipynb#X35sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m text\n",
      "\u001b[1;31mNameError\u001b[0m: name 'text' is not defined"
     ]
    }
   ],
   "source": [
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import wikipedia\n",
    "import time\n",
    "import gensim.summarization\n",
    "\n",
    "def search_and_display(query):\n",
    "    url = None\n",
    "\n",
    "    # Concatenate Wikipedia pages for the given query and extract URL of first result\n",
    "    pages = wikipedia.search(query, results=5)\n",
    "    text = ' '.join([wikipedia.page(page).content for page in pages])\n",
    "    \n",
    "    # If there is no text, return None for both metrics\n",
    "    if not text:\n",
    "        return None, None\n",
    "    \n",
    "    start_times = []\n",
    "    end_times = []\n",
    "    time_taken_to_get_urls = []\n",
    "    time_taken_to_summarize_texts = []\n",
    "    percent_words_removed_list = []\n",
    "    \n",
    "    for i in range(5):\n",
    "        # Perform Google search and extract URL of first result\n",
    "        start_time = time.time()\n",
    "        url = google_search(query)\n",
    "        end_time = time.time()\n",
    "        time_taken_to_get_urls.append(end_time - start_time)\n",
    "\n",
    "        # If URL found, extract text and generate summary\n",
    "        if url:\n",
    "            start_time = time.time()\n",
    "            summary = gensim.summarization.summarize(text)\n",
    "            end_time = time.time()\n",
    "            time_taken_to_summarize_texts.append(end_time - start_time)\n",
    "\n",
    "            # Calculate percentage of words removed in summarization\n",
    "            original_word_count = len(text.split())\n",
    "            summarized_word_count = len(summary.split())\n",
    "            percent_words_removed = (original_word_count - summarized_word_count) / original_word_count * 100\n",
    "            percent_words_removed_list.append(percent_words_removed)\n",
    "    \n",
    "    # If there are no valid metrics, return None for both\n",
    "    if not time_taken_to_get_urls or not time_taken_to_summarize_texts or not percent_words_removed_list:\n",
    "        return None, None\n",
    "    \n",
    "    # Calculate the average of all metrics\n",
    "    avg_time_taken_to_get_url = sum(time_taken_to_get_urls) / len(time_taken_to_get_urls)\n",
    "    avg_time_taken_to_summarize_text = sum(time_taken_to_summarize_texts) / len(time_taken_to_summarize_texts)\n",
    "    avg_percent_words_removed = sum(percent_words_removed_list) / len(percent_words_removed_list)\n",
    "    \n",
    "    # Print the metrics in a readable format\n",
    "    print(\"Average time taken to get URL: {:.2f} seconds\".format(avg_time_taken_to_get_url))\n",
    "    print(\"Average time taken to summarize text: {:.2f} seconds\".format(avg_time_taken_to_summarize_text))\n",
    "    print(\"Average percentage of words removed in summarization: {:.2f}%\".format(avg_percent_words_removed))\n",
    "    \n",
    "    return avg_percent_words_removed, avg_time_taken_to_get_url, avg_time_taken_to_summarize_text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average time taken to get URL: 0.55 seconds\n",
      "Average time taken to summarize text: 0.67 seconds\n",
      "Average percentage of words removed in summarization: 58.70%\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(58.69601747605444, 0.5468281745910645, 0.6677735328674317)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search_and_display(\"nlp wikipedia\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow_OIIO",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "9572e2862d927f3770936cf1299d9d1416779985874764d23793a6d1d61535d4"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
